import json
import atexit
from selenium import webdriver  # 从selenium导入webdriver
import time
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

base = 'https://www.webofscience.com'
caps = {
    'browserName': 'chrome',
    'version': '',
    'platform': 'ANY',
    'goog:loggingPrefs': {'performance': 'ALL'},  # 记录性能日志
}

driverPath = './chromedriver.exe'
driver = webdriver.Chrome(executable_path=driverPath, desired_capabilities=caps)
driver.get(base)
input("登录后请按下回车：")
urls = []

writeUrls = []


@atexit.register
def asExit():
    with open("notWriteUrls.json", 'w') as f:
        differenceSet = set(urls) - set(writeUrls)
        f.write(json.dumps(list(differenceSet)))


with open('urls.json', 'r') as f:
    urls = json.loads(f.read())

for url in urls:
    logs = driver.get_log("performance")
    driver.get(url)
    try:
        element = WebDriverWait(driver, 10).until(
            EC.presence_of_element_located((By.TAG_NAME, "app-full-record"))
        )
    except:
        time.sleep(420)
        continue
    logs = driver.get_log("performance")
    for log in logs:
        try:
            imess = json.loads(log['message'])['message']

            if imess['method'] == 'Network.webSocketFrameReceived':
                isData = False
                if imess['params']['response']['payloadData'][0] == '{':
                    imess = json.loads(
                        imess['params']['response']['payloadData'])
                    isData = True
                if isData and (imess['key'] == 'records' or imess['key'] == 'full-record'):
                    title = url.lstrip(
                        "https://www.webofscience.com/wos/alldb/full-record/").replace(':', '-')
                    with open("./result/{}.json".format(title), 'w') as f:
                        f.write(json.dumps(imess['payload']))
                        writeUrls.append(url)
        except:
            continue
